{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "25e9c222",
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "\n",
    "os.environ['CUDA_VISIBLE_DEVICES'] = '0'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "21086a1d",
   "metadata": {},
   "outputs": [],
   "source": [
    "import tensorflow as tf\n",
    "import numpy as np\n",
    "import json\n",
    "from glob import glob\n",
    "from unidecode import unidecode"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "7e416b4e",
   "metadata": {},
   "outputs": [],
   "source": [
    "# !wget https://raw.githubusercontent.com/open-dict-data/ipa-dict/master/data/ma.txt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "97d3b18f",
   "metadata": {},
   "outputs": [],
   "source": [
    "with open('ma.txt') as fopen:\n",
    "    data = fopen.read().split('\\n')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "af0a0312",
   "metadata": {},
   "outputs": [],
   "source": [
    "import re\n",
    "\n",
    "left, right = [], []\n",
    "for d in data:\n",
    "    if not len(d):\n",
    "        continue\n",
    "    splitted = d.split('\\t')\n",
    "    if not len(splitted) == 2:\n",
    "        continue\n",
    "    l = re.sub(r'[ ]+', ' ', unidecode(splitted[0])).strip()\n",
    "    r = re.sub(r'[ ]+', ' ', splitted[1]).strip()[1:-1]\n",
    "    left.append(l)\n",
    "    right.append(r)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "2f9a7d94",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['aba',\n",
       " 'ab',\n",
       " \"ab'ad\",\n",
       " 'abad',\n",
       " 'abadi',\n",
       " 'abadiah',\n",
       " 'abadikan',\n",
       " \"a'bah\",\n",
       " 'abah',\n",
       " 'abai']"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "left[:10]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "a266bcc3",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['aba',\n",
       " 'ab',\n",
       " 'abʔad',\n",
       " 'abad',\n",
       " 'abadi',\n",
       " 'abadiah',\n",
       " 'abadikan',\n",
       " 'aʔbah',\n",
       " 'abah',\n",
       " 'abai']"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "right[:10]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "07c0da52",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[0, 1, 2, 3]"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "PAD = 0\n",
    "EOS = 1\n",
    "UNK = 2\n",
    "GO = 3\n",
    "[PAD, EOS, UNK, GO]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "218fd257",
   "metadata": {},
   "outputs": [],
   "source": [
    "left_dict = [PAD, EOS, UNK, GO] + sorted(set(list(''.join(left))))\n",
    "left_dict = {c: no for no, c in enumerate(left_dict)}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "893d1157",
   "metadata": {},
   "outputs": [],
   "source": [
    "right_dict = [PAD, EOS, UNK, GO] + sorted(set(list(''.join(right))))\n",
    "right_dict = {c: no for no, c in enumerate(right_dict)}\n",
    "rev_right_dict = {v: k for k, v in right_dict.items()}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "49331bcf",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{0: 0,\n",
       " 1: 1,\n",
       " 2: 2,\n",
       " 3: 3,\n",
       " \"'\": 4,\n",
       " '-': 5,\n",
       " '.': 6,\n",
       " 'a': 7,\n",
       " 'b': 8,\n",
       " 'c': 9,\n",
       " 'd': 10,\n",
       " 'e': 11,\n",
       " 'f': 12,\n",
       " 'g': 13,\n",
       " 'h': 14,\n",
       " 'i': 15,\n",
       " 'j': 16,\n",
       " 'k': 17,\n",
       " 'l': 18,\n",
       " 'm': 19,\n",
       " 'n': 20,\n",
       " 'o': 21,\n",
       " 'p': 22,\n",
       " 'q': 23,\n",
       " 'r': 24,\n",
       " 's': 25,\n",
       " 't': 26,\n",
       " 'u': 27,\n",
       " 'v': 28,\n",
       " 'w': 29,\n",
       " 'y': 30,\n",
       " 'z': 31}"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "left_dict"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "7854c11d",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{0: 0,\n",
       " 1: 1,\n",
       " 2: 2,\n",
       " 3: 3,\n",
       " '-': 4,\n",
       " '.': 5,\n",
       " 'a': 6,\n",
       " 'b': 7,\n",
       " 'd': 8,\n",
       " 'e': 9,\n",
       " 'f': 10,\n",
       " 'g': 11,\n",
       " 'h': 12,\n",
       " 'i': 13,\n",
       " 'j': 14,\n",
       " 'k': 15,\n",
       " 'l': 16,\n",
       " 'm': 17,\n",
       " 'n': 18,\n",
       " 'o': 19,\n",
       " 'p': 20,\n",
       " 'r': 21,\n",
       " 's': 22,\n",
       " 't': 23,\n",
       " 'u': 24,\n",
       " 'v': 25,\n",
       " 'w': 26,\n",
       " 'z': 27,\n",
       " 'ŋ': 28,\n",
       " 'ə': 29,\n",
       " 'ɲ': 30,\n",
       " 'ʃ': 31,\n",
       " 'ʒ': 32,\n",
       " 'ʔ': 33}"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "right_dict"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "id": "2e77f60f",
   "metadata": {},
   "outputs": [],
   "source": [
    "class Translator:\n",
    "    def __init__(self, size_layer, num_layers, embedded_size,\n",
    "                 from_dict_size, to_dict_size, learning_rate, beam_width = 10):\n",
    "        \n",
    "        def cells(reuse=False):\n",
    "            return tf.nn.rnn_cell.LSTMCell(size_layer,initializer=tf.orthogonal_initializer(),reuse=reuse)\n",
    "        \n",
    "        def attention(encoder_out, seq_len, reuse=False):\n",
    "            attention_mechanism = tf.contrib.seq2seq.BahdanauAttention(num_units = size_layer, \n",
    "                                                                    memory = encoder_out,\n",
    "                                                                    memory_sequence_length = seq_len)\n",
    "            return tf.contrib.seq2seq.AttentionWrapper(\n",
    "            cell = tf.nn.rnn_cell.MultiRNNCell([cells(reuse) for _ in range(num_layers)]), \n",
    "                attention_mechanism = attention_mechanism,\n",
    "                attention_layer_size = size_layer)\n",
    "        \n",
    "        self.X = tf.placeholder(tf.int32, [None, None])\n",
    "        self.Y = tf.placeholder(tf.int32, [None, None])\n",
    "        self.X_seq_len = tf.count_nonzero(self.X, 1, dtype=tf.int32)\n",
    "        self.Y_seq_len = tf.count_nonzero(self.Y, 1, dtype=tf.int32)\n",
    "        batch_size = tf.shape(self.X)[0]\n",
    "        \n",
    "        encoder_embedding = tf.Variable(tf.random_uniform([from_dict_size, embedded_size], -1, 1))\n",
    "        decoder_embedding = tf.Variable(tf.random_uniform([to_dict_size, embedded_size], -1, 1))\n",
    "        \n",
    "        encoder_out, encoder_state = tf.nn.dynamic_rnn(\n",
    "            cell = tf.nn.rnn_cell.MultiRNNCell([cells() for _ in range(num_layers)]), \n",
    "            inputs = tf.nn.embedding_lookup(encoder_embedding, self.X),\n",
    "            sequence_length = self.X_seq_len,\n",
    "            dtype = tf.float32)\n",
    "        main = tf.strided_slice(self.Y, [0, 0], [batch_size, -1], [1, 1])\n",
    "        decoder_input = tf.concat([tf.fill([batch_size, 1], GO), main], 1)\n",
    "        dense = tf.layers.Dense(to_dict_size)\n",
    "        \n",
    "        with tf.variable_scope('decode'):\n",
    "            decoder_cells = attention(encoder_out, self.X_seq_len)\n",
    "            training_helper = tf.contrib.seq2seq.TrainingHelper(\n",
    "                inputs = tf.nn.embedding_lookup(decoder_embedding, decoder_input),\n",
    "                sequence_length = self.Y_seq_len,\n",
    "                time_major = False)\n",
    "            training_decoder = tf.contrib.seq2seq.BasicDecoder(\n",
    "                    cell = decoder_cells,\n",
    "                    helper = training_helper,\n",
    "                    initial_state = decoder_cells.zero_state(batch_size, tf.float32).clone(cell_state=encoder_state),\n",
    "                    output_layer = dense)\n",
    "            training_decoder_output, _, _ = tf.contrib.seq2seq.dynamic_decode(\n",
    "                    decoder = training_decoder,\n",
    "                    impute_finished = True,\n",
    "                    maximum_iterations = tf.reduce_max(self.Y_seq_len))\n",
    "            self.training_logits = training_decoder_output.rnn_output\n",
    "            \n",
    "        with tf.variable_scope('decode', reuse=True):\n",
    "            predicting_helper = tf.contrib.seq2seq.GreedyEmbeddingHelper(\n",
    "                embedding = decoder_embedding,\n",
    "                start_tokens = tf.tile(tf.constant([GO], dtype=tf.int32), [batch_size]),\n",
    "                end_token = EOS)\n",
    "            predicting_decoder = tf.contrib.seq2seq.BasicDecoder(\n",
    "                    cell = decoder_cells,\n",
    "                    helper = predicting_helper,\n",
    "                    initial_state = decoder_cells.zero_state(batch_size, tf.float32).clone(cell_state=encoder_state),\n",
    "                    output_layer = dense)\n",
    "            predicting_decoder_output, _, _ = tf.contrib.seq2seq.dynamic_decode(\n",
    "                    decoder = predicting_decoder,\n",
    "                    impute_finished = True,\n",
    "                    maximum_iterations = 2 * tf.reduce_max(self.X_seq_len))\n",
    "            self.greedy = predicting_decoder_output.sample_id\n",
    "            self.greedy = tf.identity(self.greedy,name='greedy')\n",
    "        \n",
    "        with tf.variable_scope('decode', reuse=True):\n",
    "            \n",
    "            encoder_out_tiled = tf.contrib.seq2seq.tile_batch(encoder_out, beam_width)\n",
    "            encoder_state_tiled = tf.contrib.seq2seq.tile_batch(encoder_state, beam_width)\n",
    "            X_seq_len_tiled = tf.contrib.seq2seq.tile_batch(self.X_seq_len, beam_width)\n",
    "            decoder_cell = attention(encoder_out_tiled, X_seq_len_tiled, reuse=True)\n",
    "            \n",
    "            predicting_decoder = tf.contrib.seq2seq.BeamSearchDecoder(\n",
    "                cell = decoder_cell,\n",
    "                embedding = decoder_embedding,\n",
    "                start_tokens = tf.tile(tf.constant([GO], dtype=tf.int32), [batch_size]),\n",
    "                end_token = EOS,\n",
    "                initial_state = decoder_cell.zero_state(batch_size * beam_width, tf.float32).clone(\n",
    "                    cell_state = encoder_state_tiled),\n",
    "                beam_width = beam_width,\n",
    "                output_layer = dense,\n",
    "                length_penalty_weight = 0.0)\n",
    "            \n",
    "            predicting_decoder_output, _, _ = tf.contrib.seq2seq.dynamic_decode(\n",
    "                decoder = predicting_decoder,\n",
    "                impute_finished = False,\n",
    "                maximum_iterations = tf.reduce_max(self.X_seq_len))\n",
    "            \n",
    "            self.beam = predicting_decoder_output.predicted_ids[:, :, 0]\n",
    "            self.beam = tf.identity(self.beam,name='beam')\n",
    "        \n",
    "        masks = tf.sequence_mask(self.Y_seq_len, tf.reduce_max(self.Y_seq_len), dtype=tf.float32)\n",
    "        self.masks = masks\n",
    "        self.cost = tf.contrib.seq2seq.sequence_loss(logits = self.training_logits,\n",
    "                                                     targets = self.Y,\n",
    "                                                     weights = masks)\n",
    "        self.optimizer = tf.train.AdamOptimizer(learning_rate = learning_rate).minimize(self.cost)\n",
    "        y_t = tf.argmax(self.training_logits,axis=2)\n",
    "        y_t = tf.cast(y_t, tf.int32)\n",
    "        self.prediction = tf.boolean_mask(y_t, masks)\n",
    "        mask_label = tf.boolean_mask(self.Y, masks)\n",
    "        correct_pred = tf.equal(self.prediction, mask_label)\n",
    "        correct_index = tf.cast(correct_pred, tf.float32)\n",
    "        self.accuracy = tf.reduce_mean(tf.cast(correct_pred, tf.float32))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "id": "41934452",
   "metadata": {},
   "outputs": [],
   "source": [
    "size_layer = 256\n",
    "num_layers = 2\n",
    "embedded_size = 256\n",
    "learning_rate = 1e-3\n",
    "batch_size = 32\n",
    "epoch = 20"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "id": "737dd359",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From /tmp/ipykernel_5482/221865152.py:1: The name tf.reset_default_graph is deprecated. Please use tf.compat.v1.reset_default_graph instead.\n",
      "\n",
      "WARNING:tensorflow:From /tmp/ipykernel_5482/221865152.py:2: The name tf.InteractiveSession is deprecated. Please use tf.compat.v1.InteractiveSession instead.\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2022-07-03 17:27:59.230380: I tensorflow/core/platform/profile_utils/cpu_utils.cc:109] CPU Frequency: 2496000000 Hz\n",
      "2022-07-03 17:27:59.231325: I tensorflow/compiler/xla/service/service.cc:168] XLA service 0x1447050 initialized for platform Host (this does not guarantee that XLA will be used). Devices:\n",
      "2022-07-03 17:27:59.231338: I tensorflow/compiler/xla/service/service.cc:176]   StreamExecutor device (0): Host, Default Version\n",
      "2022-07-03 17:27:59.233274: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcuda.so.1\n",
      "2022-07-03 17:27:59.332792: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:1082] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero\n",
      "2022-07-03 17:27:59.333590: I tensorflow/compiler/xla/service/service.cc:168] XLA service 0x139d1f0 initialized for platform CUDA (this does not guarantee that XLA will be used). Devices:\n",
      "2022-07-03 17:27:59.333602: I tensorflow/compiler/xla/service/service.cc:176]   StreamExecutor device (0): NVIDIA GeForce RTX 3090 Ti, Compute Capability 8.6\n",
      "2022-07-03 17:27:59.333691: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:1082] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero\n",
      "2022-07-03 17:27:59.334433: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1666] Found device 0 with properties: \n",
      "name: NVIDIA GeForce RTX 3090 Ti major: 8 minor: 6 memoryClockRate(GHz): 1.86\n",
      "pciBusID: 0000:01:00.0\n",
      "2022-07-03 17:27:59.334466: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudart.so.11.0\n",
      "2022-07-03 17:27:59.351280: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcublas.so.11\n",
      "2022-07-03 17:27:59.372475: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcufft.so.10\n",
      "2022-07-03 17:27:59.374872: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcurand.so.10\n",
      "2022-07-03 17:27:59.377817: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcusolver.so.11\n",
      "2022-07-03 17:27:59.383216: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcusparse.so.11\n",
      "2022-07-03 17:27:59.383679: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudnn.so.8\n",
      "2022-07-03 17:27:59.383770: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:1082] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero\n",
      "2022-07-03 17:27:59.384549: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:1082] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero\n",
      "2022-07-03 17:27:59.385252: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1794] Adding visible gpu devices: 0\n",
      "2022-07-03 17:27:59.385568: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudart.so.11.0\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From /tmp/ipykernel_5482/2944119294.py:17: The name tf.placeholder is deprecated. Please use tf.compat.v1.placeholder instead.\n",
      "\n",
      "WARNING:tensorflow:From /tmp/ipykernel_5482/2944119294.py:23: The name tf.random_uniform is deprecated. Please use tf.random.uniform instead.\n",
      "\n",
      "WARNING:tensorflow:From /tmp/ipykernel_5482/2944119294.py:33: The name tf.layers.Dense is deprecated. Please use tf.compat.v1.layers.Dense instead.\n",
      "\n",
      "WARNING:tensorflow:From /tmp/ipykernel_5482/2944119294.py:35: The name tf.variable_scope is deprecated. Please use tf.compat.v1.variable_scope instead.\n",
      "\n",
      "WARNING:tensorflow:\n",
      "The TensorFlow contrib module will not be included in TensorFlow 2.0.\n",
      "For more information, please see:\n",
      "  * https://github.com/tensorflow/community/blob/master/rfcs/20180907-contrib-sunset.md\n",
      "  * https://github.com/tensorflow/addons\n",
      "  * https://github.com/tensorflow/io (for I/O related ops)\n",
      "If you depend on functionality not listed there, please file an issue.\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2022-07-03 17:28:00.066621: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1206] Device interconnect StreamExecutor with strength 1 edge matrix:\n",
      "2022-07-03 17:28:00.066642: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1212]      0 \n",
      "2022-07-03 17:28:00.066646: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1225] 0:   N \n",
      "2022-07-03 17:28:00.067079: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:1082] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero\n",
      "2022-07-03 17:28:00.067873: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:1082] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero\n",
      "2022-07-03 17:28:00.068601: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1351] Created TensorFlow device (/job:localhost/replica:0/task:0/device:GPU:0 with 22047 MB memory) -> physical GPU (device: 0, name: NVIDIA GeForce RTX 3090 Ti, pci bus id: 0000:01:00.0, compute capability: 8.6)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From /tmp/ipykernel_5482/2944119294.py:100: The name tf.train.AdamOptimizer is deprecated. Please use tf.compat.v1.train.AdamOptimizer instead.\n",
      "\n",
      "WARNING:tensorflow:From /tmp/ipykernel_5482/221865152.py:4: The name tf.global_variables_initializer is deprecated. Please use tf.compat.v1.global_variables_initializer instead.\n",
      "\n"
     ]
    }
   ],
   "source": [
    "tf.reset_default_graph()\n",
    "sess = tf.InteractiveSession()\n",
    "model = Translator(size_layer, num_layers, embedded_size, len(left_dict), len(right_dict), learning_rate)\n",
    "sess.run(tf.global_variables_initializer())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "id": "05e06142",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From /tmp/ipykernel_5482/3508881337.py:1: The name tf.train.Saver is deprecated. Please use tf.compat.v1.train.Saver instead.\n",
      "\n",
      "WARNING:tensorflow:From /tmp/ipykernel_5482/3508881337.py:1: The name tf.trainable_variables is deprecated. Please use tf.compat.v1.trainable_variables instead.\n",
      "\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "'lstm/model.ckpt'"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "saver = tf.train.Saver(tf.trainable_variables())\n",
    "saver.save(sess, 'lstm/model.ckpt')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "id": "9761e43d",
   "metadata": {},
   "outputs": [],
   "source": [
    "def pad_sentence_batch(sentence_batch, pad_int):\n",
    "    padded_seqs = []\n",
    "    seq_lens = []\n",
    "    max_sentence_len = max([len(sentence) for sentence in sentence_batch])\n",
    "    for sentence in sentence_batch:\n",
    "        padded_seqs.append(sentence + [pad_int] * (max_sentence_len - len(sentence)))\n",
    "        seq_lens.append(len(sentence))\n",
    "    return padded_seqs, seq_lens"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "id": "d758d008",
   "metadata": {},
   "outputs": [],
   "source": [
    "train_X, train_Y = [], []\n",
    "for i in range(len(left)):\n",
    "    train_X.append([left_dict[c] for c in left[i]] + [1])\n",
    "    train_Y.append([right_dict[c] for c in right[i]] + [1])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "id": "6908878c",
   "metadata": {},
   "outputs": [],
   "source": [
    "batch_x, _ = pad_sentence_batch(train_X[: 5], 0)\n",
    "batch_y, _ = pad_sentence_batch(train_Y[: 5], 0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "id": "5c79626a",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(5, 6)"
      ]
     },
     "execution_count": 29,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.array(batch_y).shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "id": "8ffaa31d",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2022-07-03 17:28:27.297797: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcublas.so.11\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 547 ms, sys: 165 ms, total: 712 ms\n",
      "Wall time: 1.06 s\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "[3.5310547, None]"
      ]
     },
     "execution_count": 30,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "%%time\n",
    "\n",
    "sess.run([model.cost, model.optimizer], feed_dict = {model.X: batch_x, model.Y: batch_y})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "id": "5c1593ab",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "minibatch loop: 100%|████████████████████████████████████████████████| 882/882 [00:24<00:00, 35.57it/s, accuracy=0.995, cost=0.00765]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch 1, training avg loss 0.022205, training avg acc 0.991366\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "minibatch loop: 100%|██████████████████████████████████████████████████| 882/882 [00:23<00:00, 38.24it/s, accuracy=0.99, cost=0.0247]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch 2, training avg loss 0.022535, training avg acc 0.991475\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "minibatch loop: 100%|█████████████████████████████████████████████████| 882/882 [00:22<00:00, 38.70it/s, accuracy=0.984, cost=0.0306]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch 3, training avg loss 0.019012, training avg acc 0.992442\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "import tqdm\n",
    "from sklearn.utils import shuffle\n",
    "\n",
    "for e in range(3):\n",
    "    train_X, train_Y = shuffle(train_X, train_Y)\n",
    "    pbar = tqdm.tqdm(\n",
    "        range(0, len(train_X), batch_size), desc = 'minibatch loop')\n",
    "    train_loss, train_acc = [], []\n",
    "    for i in pbar:\n",
    "        index = min(i + batch_size, len(train_X))\n",
    "        batch_x, seq_x = pad_sentence_batch(train_X[i : index], PAD)\n",
    "        batch_y, seq_y = pad_sentence_batch(train_Y[i : index], PAD)\n",
    "        feed = {model.X: batch_x,\n",
    "                model.Y: batch_y}\n",
    "        accuracy, loss, _ = sess.run([model.accuracy,model.cost,model.optimizer],\n",
    "                                    feed_dict = feed)\n",
    "        train_loss.append(loss)\n",
    "        train_acc.append(accuracy)\n",
    "        pbar.set_postfix(cost = loss, accuracy = accuracy)\n",
    "    \n",
    "    print('epoch %d, training avg loss %f, training avg acc %f'%(e+1,\n",
    "                                                                 np.mean(train_loss),np.mean(train_acc)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "id": "3d230f5f",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'lstm-bahdanau/model.ckpt'"
      ]
     },
     "execution_count": 37,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "saver = tf.train.Saver(tf.trainable_variables())\n",
    "saver.save(sess, 'lstm-bahdanau/model.ckpt')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "id": "845efbf6",
   "metadata": {},
   "outputs": [],
   "source": [
    "string = 'comel'\n",
    "batch = [left_dict[c] for c in string] + [1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "id": "d6964155",
   "metadata": {},
   "outputs": [],
   "source": [
    "greedy, beam = sess.run([model.greedy, model.beam], feed_dict = {model.X: [batch]})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "id": "266f736b",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'tʃomel'"
      ]
     },
     "execution_count": 40,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "''.join([rev_right_dict[i] for i in greedy[0] if i > 3])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "id": "fd34c47d",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From /tmp/ipykernel_5482/3818740975.py:4: The name tf.get_default_graph is deprecated. Please use tf.compat.v1.get_default_graph instead.\n",
      "\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "['Placeholder',\n",
       " 'Placeholder_1',\n",
       " 'Variable',\n",
       " 'Variable_1',\n",
       " 'rnn/multi_rnn_cell/cell_0/lstm_cell/kernel',\n",
       " 'rnn/multi_rnn_cell/cell_0/lstm_cell/bias',\n",
       " 'rnn/multi_rnn_cell/cell_1/lstm_cell/kernel',\n",
       " 'rnn/multi_rnn_cell/cell_1/lstm_cell/bias',\n",
       " 'decode/memory_layer/kernel',\n",
       " 'decode/decoder/attention_wrapper/multi_rnn_cell/cell_0/lstm_cell/kernel',\n",
       " 'decode/decoder/attention_wrapper/multi_rnn_cell/cell_0/lstm_cell/bias',\n",
       " 'decode/decoder/attention_wrapper/multi_rnn_cell/cell_1/lstm_cell/kernel',\n",
       " 'decode/decoder/attention_wrapper/multi_rnn_cell/cell_1/lstm_cell/bias',\n",
       " 'decode/decoder/attention_wrapper/bahdanau_attention/query_layer/kernel',\n",
       " 'decode/decoder/attention_wrapper/bahdanau_attention/attention_v',\n",
       " 'decode/decoder/attention_wrapper/attention_layer/kernel',\n",
       " 'decode/decoder/dense/kernel',\n",
       " 'decode/decoder/dense/bias',\n",
       " 'decode_1/greedy',\n",
       " 'decode_2/decoder/while/BeamSearchDecoderStep/beam_width',\n",
       " 'decode_2/decoder/while/BeamSearchDecoderStep/next_beam_probs/range/start',\n",
       " 'decode_2/decoder/while/BeamSearchDecoderStep/next_beam_probs/range/delta',\n",
       " 'decode_2/decoder/while/BeamSearchDecoderStep/next_beam_probs/range',\n",
       " 'decode_2/decoder/while/BeamSearchDecoderStep/next_beam_probs/mul/y',\n",
       " 'decode_2/decoder/while/BeamSearchDecoderStep/next_beam_probs/mul',\n",
       " 'decode_2/decoder/while/BeamSearchDecoderStep/next_beam_probs/ExpandDims/dim',\n",
       " 'decode_2/decoder/while/BeamSearchDecoderStep/next_beam_probs/ExpandDims',\n",
       " 'decode_2/decoder/while/BeamSearchDecoderStep/next_beam_probs/add',\n",
       " 'decode_2/decoder/while/BeamSearchDecoderStep/next_beam_probs/Reshape/shape',\n",
       " 'decode_2/decoder/while/BeamSearchDecoderStep/next_beam_probs/Reshape',\n",
       " 'decode_2/decoder/while/BeamSearchDecoderStep/next_beam_probs/Reshape_1/shape',\n",
       " 'decode_2/decoder/while/BeamSearchDecoderStep/next_beam_probs/Reshape_1',\n",
       " 'decode_2/decoder/while/BeamSearchDecoderStep/next_beam_probs/GatherV2/axis',\n",
       " 'decode_2/decoder/while/BeamSearchDecoderStep/next_beam_probs/GatherV2',\n",
       " 'decode_2/decoder/while/BeamSearchDecoderStep/next_beam_probs/Shape',\n",
       " 'decode_2/decoder/while/BeamSearchDecoderStep/next_beam_probs/strided_slice/stack',\n",
       " 'decode_2/decoder/while/BeamSearchDecoderStep/next_beam_probs/strided_slice/stack_1',\n",
       " 'decode_2/decoder/while/BeamSearchDecoderStep/next_beam_probs/strided_slice/stack_2',\n",
       " 'decode_2/decoder/while/BeamSearchDecoderStep/next_beam_probs/strided_slice',\n",
       " 'decode_2/decoder/while/BeamSearchDecoderStep/next_beam_probs/output',\n",
       " 'decode_2/decoder/while/BeamSearchDecoderStep/next_beam_word_ids/y',\n",
       " 'decode_2/decoder/while/BeamSearchDecoderStep/next_beam_word_ids',\n",
       " 'decode_2/decoder/while/BeamSearchDecoderStep/next_beam_parent_ids',\n",
       " 'decode_2/decoder/while/BeamSearchDecoderStep/next_beam_finished',\n",
       " 'decode_2/beam']"
      ]
     },
     "execution_count": 41,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "strings = ','.join(\n",
    "    [\n",
    "        n.name\n",
    "        for n in tf.get_default_graph().as_graph_def().node\n",
    "        if ('Variable' in n.op\n",
    "        or 'Placeholder' in n.name\n",
    "        or 'greedy' in n.name\n",
    "        or 'beam' in n.name\n",
    "        or 'alphas' in n.name)\n",
    "        and 'Adam' not in n.name\n",
    "        and 'beta' not in n.name\n",
    "        and 'OptimizeLoss' not in n.name\n",
    "        and 'Global_Step' not in n.name\n",
    "    ]\n",
    ")\n",
    "strings.split(',')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "id": "c4260ce7",
   "metadata": {},
   "outputs": [],
   "source": [
    "def freeze_graph(model_dir, output_node_names):\n",
    "\n",
    "    if not tf.gfile.Exists(model_dir):\n",
    "        raise AssertionError(\n",
    "            \"Export directory doesn't exists. Please specify an export \"\n",
    "            \"directory: %s\" % model_dir)\n",
    "\n",
    "    checkpoint = tf.train.get_checkpoint_state(model_dir)\n",
    "    input_checkpoint = checkpoint.model_checkpoint_path\n",
    "    \n",
    "    absolute_model_dir = \"/\".join(input_checkpoint.split('/')[:-1])\n",
    "    output_graph = absolute_model_dir + \"/frozen_model.pb\"\n",
    "    clear_devices = True\n",
    "    with tf.Session(graph=tf.Graph()) as sess:\n",
    "        saver = tf.train.import_meta_graph(input_checkpoint + '.meta', clear_devices=clear_devices)\n",
    "        saver.restore(sess, input_checkpoint)\n",
    "        output_graph_def = tf.graph_util.convert_variables_to_constants(\n",
    "            sess,\n",
    "            tf.get_default_graph().as_graph_def(),\n",
    "            output_node_names.split(\",\")\n",
    "        ) \n",
    "        with tf.gfile.GFile(output_graph, \"wb\") as f:\n",
    "            f.write(output_graph_def.SerializeToString())\n",
    "        print(\"%d ops in the final graph.\" % len(output_graph_def.node))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "id": "e97889fb",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From /tmp/ipykernel_5482/1070649395.py:3: The name tf.gfile.Exists is deprecated. Please use tf.io.gfile.exists instead.\n",
      "\n",
      "WARNING:tensorflow:From /tmp/ipykernel_5482/1070649395.py:14: The name tf.Session is deprecated. Please use tf.compat.v1.Session instead.\n",
      "\n",
      "WARNING:tensorflow:From /tmp/ipykernel_5482/1070649395.py:15: The name tf.train.import_meta_graph is deprecated. Please use tf.compat.v1.train.import_meta_graph instead.\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2022-07-03 17:34:41.604615: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:1082] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero\n",
      "2022-07-03 17:34:41.605601: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1666] Found device 0 with properties: \n",
      "name: NVIDIA GeForce RTX 3090 Ti major: 8 minor: 6 memoryClockRate(GHz): 1.86\n",
      "pciBusID: 0000:01:00.0\n",
      "2022-07-03 17:34:41.605623: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudart.so.11.0\n",
      "2022-07-03 17:34:41.605633: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcublas.so.11\n",
      "2022-07-03 17:34:41.605638: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcufft.so.10\n",
      "2022-07-03 17:34:41.605642: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcurand.so.10\n",
      "2022-07-03 17:34:41.605646: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcusolver.so.11\n",
      "2022-07-03 17:34:41.605651: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcusparse.so.11\n",
      "2022-07-03 17:34:41.605655: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudnn.so.8\n",
      "2022-07-03 17:34:41.605687: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:1082] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero\n",
      "2022-07-03 17:34:41.606414: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:1082] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero\n",
      "2022-07-03 17:34:41.608494: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1794] Adding visible gpu devices: 0\n",
      "2022-07-03 17:34:41.610352: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:1082] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero\n",
      "2022-07-03 17:34:41.613895: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1666] Found device 0 with properties: \n",
      "name: NVIDIA GeForce RTX 3090 Ti major: 8 minor: 6 memoryClockRate(GHz): 1.86\n",
      "pciBusID: 0000:01:00.0\n",
      "2022-07-03 17:34:41.613959: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudart.so.11.0\n",
      "2022-07-03 17:34:41.613994: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcublas.so.11\n",
      "2022-07-03 17:34:41.614022: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcufft.so.10\n",
      "2022-07-03 17:34:41.614040: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcurand.so.10\n",
      "2022-07-03 17:34:41.614059: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcusolver.so.11\n",
      "2022-07-03 17:34:41.614077: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcusparse.so.11\n",
      "2022-07-03 17:34:41.614097: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudnn.so.8\n",
      "2022-07-03 17:34:41.614186: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:1082] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero\n",
      "2022-07-03 17:34:41.616127: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:1082] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero\n",
      "2022-07-03 17:34:41.616824: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1794] Adding visible gpu devices: 0\n",
      "2022-07-03 17:34:41.616839: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1206] Device interconnect StreamExecutor with strength 1 edge matrix:\n",
      "2022-07-03 17:34:41.616842: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1212]      0 \n",
      "2022-07-03 17:34:41.616845: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1225] 0:   N \n",
      "2022-07-03 17:34:41.616880: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:1082] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero\n",
      "2022-07-03 17:34:41.617588: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:1082] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero\n",
      "2022-07-03 17:34:41.618316: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1351] Created TensorFlow device (/job:localhost/replica:0/task:0/device:GPU:0 with 22047 MB memory) -> physical GPU (device: 0, name: NVIDIA GeForce RTX 3090 Ti, pci bus id: 0000:01:00.0, compute capability: 8.6)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Restoring parameters from lstm-bahdanau/model.ckpt\n",
      "INFO:tensorflow:Froze 16 variables.\n",
      "INFO:tensorflow:Converted 16 variables to const ops.\n",
      "WARNING:tensorflow:From /tmp/ipykernel_5482/1070649395.py:22: The name tf.gfile.GFile is deprecated. Please use tf.io.gfile.GFile instead.\n",
      "\n",
      "1649 ops in the final graph.\n"
     ]
    }
   ],
   "source": [
    "freeze_graph(\"lstm-bahdanau\", strings)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "id": "120bfcf1",
   "metadata": {},
   "outputs": [],
   "source": [
    "def load_graph(frozen_graph_filename):\n",
    "    with tf.gfile.GFile(frozen_graph_filename, \"rb\") as f:\n",
    "        graph_def = tf.GraphDef()\n",
    "        graph_def.ParseFromString(f.read())\n",
    "    with tf.Graph().as_default() as graph:\n",
    "        tf.import_graph_def(graph_def)\n",
    "    return graph"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "id": "da030faa",
   "metadata": {},
   "outputs": [],
   "source": [
    "from tensorflow.tools.graph_transforms import TransformGraph\n",
    "from tensorflow.contrib.seq2seq.python.ops import beam_search_ops"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "id": "9955eaa1",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From /tmp/ipykernel_5482/295500348.py:10: The name tf.GraphDef is deprecated. Please use tf.compat.v1.GraphDef instead.\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2022-07-03 17:34:53.569589: I tensorflow/tools/graph_transforms/transform_graph.cc:318] Applying add_default_attributes\n",
      "2022-07-03 17:34:53.575625: I tensorflow/tools/graph_transforms/transform_graph.cc:318] Applying remove_nodes\n",
      "2022-07-03 17:34:53.581901: I tensorflow/tools/graph_transforms/remove_nodes.cc:78] Skipping replacement for decode_1/greedy\n",
      "2022-07-03 17:34:53.582822: I tensorflow/tools/graph_transforms/remove_nodes.cc:78] Skipping replacement for decode_2/beam\n",
      "2022-07-03 17:34:53.590458: I tensorflow/tools/graph_transforms/remove_nodes.cc:78] Skipping replacement for decode_1/greedy\n",
      "2022-07-03 17:34:53.591129: I tensorflow/tools/graph_transforms/remove_nodes.cc:78] Skipping replacement for decode_2/beam\n",
      "2022-07-03 17:34:53.597858: I tensorflow/tools/graph_transforms/remove_nodes.cc:78] Skipping replacement for decode_1/greedy\n",
      "2022-07-03 17:34:53.598495: I tensorflow/tools/graph_transforms/remove_nodes.cc:78] Skipping replacement for decode_2/beam\n",
      "2022-07-03 17:34:53.618696: I tensorflow/tools/graph_transforms/transform_graph.cc:318] Applying fold_batch_norms\n",
      "2022-07-03 17:34:53.628453: I tensorflow/tools/graph_transforms/transform_graph.cc:318] Applying fold_old_batch_norms\n",
      "2022-07-03 17:34:53.653686: I tensorflow/tools/graph_transforms/transform_graph.cc:318] Applying quantize_weights\n",
      "2022-07-03 17:34:53.692165: I tensorflow/tools/graph_transforms/transform_graph.cc:318] Applying strip_unused_nodes\n",
      "2022-07-03 17:34:53.698123: I tensorflow/tools/graph_transforms/transform_graph.cc:318] Applying sort_by_execution_order\n"
     ]
    }
   ],
   "source": [
    "transforms = ['add_default_attributes',\n",
    "             'remove_nodes(op=Identity, op=CheckNumerics, op=Dropout)',\n",
    "             'fold_batch_norms',\n",
    "             'fold_old_batch_norms',\n",
    "             'quantize_weights(fallback_min=-10, fallback_max=10)',\n",
    "             'strip_unused_nodes',\n",
    "             'sort_by_execution_order']\n",
    "\n",
    "pb = 'lstm-bahdanau/frozen_model.pb'\n",
    "input_graph_def = tf.GraphDef()\n",
    "with tf.gfile.FastGFile(pb, 'rb') as f:\n",
    "    input_graph_def.ParseFromString(f.read())\n",
    "\n",
    "transformed_graph_def = TransformGraph(input_graph_def, \n",
    "                                       ['Placeholder'],\n",
    "                                       ['decode_1/greedy', 'decode_2/beam'], transforms)\n",
    "\n",
    "with tf.gfile.GFile(f'{pb}.quantized', 'wb') as f:\n",
    "    f.write(transformed_graph_def.SerializeToString())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "id": "8fa9e7d2",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "lstm-bahdanau/\r\n",
      "lstm-bahdanau/checkpoint\r\n",
      "lstm-bahdanau/frozen_model.pb.quantized\r\n",
      "lstm-bahdanau/model.ckpt.index\r\n",
      "lstm-bahdanau/model.ckpt.data-00000-of-00001\r\n",
      "lstm-bahdanau/model.ckpt.meta\r\n",
      "lstm-bahdanau/frozen_model.pb\r\n"
     ]
    }
   ],
   "source": [
    "!tar -cvf phoneme-ipa-lstm-bahdanau.tar lstm-bahdanau"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "id": "75571830",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/ubuntu/tf-nvidia/lib/python3.8/site-packages/tqdm/auto.py:22: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n",
      "  from .autonotebook import tqdm as notebook_tqdm\n"
     ]
    }
   ],
   "source": [
    "from malaya_boilerplate.huggingface import upload_dict"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "id": "59c6e9ff",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/ubuntu/tf-nvidia/lib/python3.8/site-packages/huggingface_hub/hf_api.py:79: FutureWarning: `name` and `organization` input arguments are deprecated and will be removed in v0.8. Pass `repo_id` instead.\n",
      "  warnings.warn(\n"
     ]
    }
   ],
   "source": [
    "files_mapping = {'lstm-bahdanau/frozen_model.pb': 'model.pb'}\n",
    "upload_dict(model = 'phoneme-ipa-lstm-bahdanau', files_mapping = files_mapping)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "id": "7705b510",
   "metadata": {},
   "outputs": [],
   "source": [
    "files_mapping = {'lstm-bahdanau/frozen_model.pb.quantized': 'model.pb'}\n",
    "upload_dict(model = 'phoneme-ipa-lstm-bahdanau-quantized', files_mapping = files_mapping)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "id": "56509a42",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "409 Client Error: Conflict for url: https://huggingface.co/api/repos/create - You already created this model repo\n"
     ]
    }
   ],
   "source": [
    "files_mapping = {'phoneme-ipa-lstm-bahdanau.tar': 'phoneme-ipa-lstm-bahdanau.tar'}\n",
    "upload_dict(model = 'pretrained-phoneme', files_mapping = files_mapping)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "tf1",
   "language": "python",
   "name": "tf1"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.10"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
